{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['cifar-10-python.tar.gz']\n"
     ]
    }
   ],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "import os\n",
    "print(os.listdir(\"../input\"))\n",
    "\n",
    "import time\n",
    "\n",
    "# import pytorch\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.optim import SGD,Adam,lr_scheduler\n",
    "from torch.utils.data import random_split\n",
    "import torchvision\n",
    "from torchvision import transforms, datasets\n",
    "from torch.utils.data import DataLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define transformations for train\n",
    "train_transform = transforms.Compose([\n",
    "    transforms.RandomHorizontalFlip(p=.40),\n",
    "    transforms.RandomRotation(30),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])])\n",
    "\n",
    "# define transformations for test\n",
    "test_transform = transforms.Compose([\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])])\n",
    "\n",
    "# define training dataloader\n",
    "def get_training_dataloader(train_transform, batch_size=128, num_workers=0, shuffle=True):\n",
    "    \"\"\" return training dataloader\n",
    "    Args:\n",
    "        train_transform: transfroms for train dataset\n",
    "        path: path to cifar100 training python dataset\n",
    "        batch_size: dataloader batchsize\n",
    "        num_workers: dataloader num_works\n",
    "        shuffle: whether to shuffle \n",
    "    Returns: train_data_loader:torch dataloader object\n",
    "    \"\"\"\n",
    "\n",
    "    transform_train = train_transform\n",
    "    cifar10_training = torchvision.datasets.CIFAR10(root='.', train=True, download=True, transform=transform_train)\n",
    "    cifar10_training_loader = DataLoader(\n",
    "        cifar10_training, shuffle=shuffle, num_workers=num_workers, batch_size=batch_size)\n",
    "\n",
    "    return cifar10_training_loader\n",
    "\n",
    "# define test dataloader\n",
    "def get_testing_dataloader(test_transform, batch_size=128, num_workers=0, shuffle=True):\n",
    "    \"\"\" return training dataloader\n",
    "    Args:\n",
    "        test_transform: transforms for test dataset\n",
    "        path: path to cifar100 test python dataset\n",
    "        batch_size: dataloader batchsize\n",
    "        num_workers: dataloader num_works\n",
    "        shuffle: whether to shuffle \n",
    "    Returns: cifar100_test_loader:torch dataloader object\n",
    "    \"\"\"\n",
    "\n",
    "    transform_test = test_transform\n",
    "    cifar10_test = torchvision.datasets.CIFAR10(root='.', train=False, download=True, transform=transform_test)\n",
    "    cifar10_test_loader = DataLoader(\n",
    "        cifar10_test, shuffle=shuffle, num_workers=num_workers, batch_size=batch_size)\n",
    "\n",
    "    return cifar10_test_loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# implement mish activation function\n",
    "def f_mish(input):\n",
    "    '''\n",
    "    Applies the mish function element-wise:\n",
    "    mish(x) = x * tanh(softplus(x)) = x * tanh(ln(1 + exp(x)))\n",
    "    '''\n",
    "    return input * torch.tanh(F.softplus(input))\n",
    "\n",
    "# implement class wrapper for mish activation function\n",
    "class mish(nn.Module):\n",
    "    '''\n",
    "    Applies the mish function element-wise:\n",
    "    mish(x) = x * tanh(softplus(x)) = x * tanh(ln(1 + exp(x)))\n",
    "\n",
    "    Shape:\n",
    "        - Input: (N, *) where * means, any number of additional\n",
    "          dimensions\n",
    "        - Output: (N, *), same shape as the input\n",
    "\n",
    "    Examples:\n",
    "        >>> m = mish()\n",
    "        >>> input = torch.randn(2)\n",
    "        >>> output = m(input)\n",
    "\n",
    "    '''\n",
    "    def __init__(self):\n",
    "        '''\n",
    "        Init method.\n",
    "        '''\n",
    "        super().__init__()\n",
    "\n",
    "    def forward(self, input):\n",
    "        '''\n",
    "        Forward pass of the function.\n",
    "        '''\n",
    "        return f_mish(input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# implement swish activation function\n",
    "def f_swish(input):\n",
    "    '''\n",
    "    Applies the swish function element-wise:\n",
    "    swish(x) = x * sigmoid(x)\n",
    "    '''\n",
    "    return input * torch.sigmoid(input)\n",
    "\n",
    "# implement class wrapper for swish activation function\n",
    "class swish(nn.Module):\n",
    "    '''\n",
    "    Applies the swish function element-wise:\n",
    "    swish(x) = x * sigmoid(x)\n",
    "\n",
    "    Shape:\n",
    "        - Input: (N, *) where * means, any number of additional\n",
    "          dimensions\n",
    "        - Output: (N, *), same shape as the input\n",
    "\n",
    "    Examples:\n",
    "        >>> m = swish()\n",
    "        >>> input = torch.randn(2)\n",
    "        >>> output = m(input)\n",
    "\n",
    "    '''\n",
    "    def __init__(self):\n",
    "        '''\n",
    "        Init method.\n",
    "        '''\n",
    "        super().__init__()\n",
    "\n",
    "    def forward(self, input):\n",
    "        '''\n",
    "        Forward pass of the function.\n",
    "        '''\n",
    "        return f_swish(input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Fire(nn.Module):\n",
    "\n",
    "    def __init__(self, in_channel, out_channel, squzee_channel, activation = 'relu'):\n",
    "\n",
    "        super().__init__()\n",
    "        \n",
    "        if activation == 'relu':\n",
    "            f_activation = nn.ReLU(inplace=True)\n",
    "            \n",
    "        if activation == 'swish':\n",
    "            f_activation = swish()\n",
    "            \n",
    "        if activation == 'mish':\n",
    "            f_activation = mish()\n",
    "        \n",
    "        self.squeeze = nn.Sequential(\n",
    "            nn.Conv2d(in_channel, squzee_channel, 1),\n",
    "            nn.BatchNorm2d(squzee_channel),\n",
    "            f_activation\n",
    "        )\n",
    "\n",
    "        self.expand_1x1 = nn.Sequential(\n",
    "            nn.Conv2d(squzee_channel, int(out_channel / 2), 1),\n",
    "            nn.BatchNorm2d(int(out_channel / 2)),\n",
    "            f_activation\n",
    "        )\n",
    "\n",
    "        self.expand_3x3 = nn.Sequential(\n",
    "            nn.Conv2d(squzee_channel, int(out_channel / 2), 3, padding=1),\n",
    "            nn.BatchNorm2d(int(out_channel / 2)),\n",
    "            f_activation\n",
    "        )\n",
    "    \n",
    "    def forward(self, x):\n",
    "\n",
    "        x = self.squeeze(x)\n",
    "        x = torch.cat([\n",
    "            self.expand_1x1(x),\n",
    "            self.expand_3x3(x)\n",
    "        ], 1)\n",
    "\n",
    "        return x\n",
    "\n",
    "class SqueezeNet(nn.Module):\n",
    "\n",
    "    \"\"\"mobile net with simple bypass\"\"\"\n",
    "    def __init__(self, class_num=10, activation = 'relu'):\n",
    "        \n",
    "        if activation == 'relu':\n",
    "            f_activation = nn.ReLU(inplace=True)\n",
    "            \n",
    "        if activation == 'swish':\n",
    "            f_activation = swish()\n",
    "            \n",
    "        if activation == 'mish':\n",
    "            f_activation = mish()\n",
    "\n",
    "        super().__init__()\n",
    "        self.stem = nn.Sequential(\n",
    "            nn.Conv2d(3, 96, 3, padding=1),\n",
    "            nn.BatchNorm2d(96),\n",
    "            f_activation,\n",
    "            nn.MaxPool2d(2, 2)\n",
    "        )\n",
    "\n",
    "        self.fire2 = Fire(96, 128, 16, activation = activation)\n",
    "        self.fire3 = Fire(128, 128, 16, activation = activation)\n",
    "        self.fire4 = Fire(128, 256, 32, activation = activation)\n",
    "        self.fire5 = Fire(256, 256, 32, activation = activation)\n",
    "        self.fire6 = Fire(256, 384, 48, activation = activation)\n",
    "        self.fire7 = Fire(384, 384, 48, activation = activation)\n",
    "        self.fire8 = Fire(384, 512, 64, activation = activation)\n",
    "        self.fire9 = Fire(512, 512, 64, activation = activation)\n",
    "\n",
    "        self.conv10 = nn.Conv2d(512, class_num, 1)\n",
    "        self.avg = nn.AdaptiveAvgPool2d(1)\n",
    "        self.maxpool = nn.MaxPool2d(2, 2)\n",
    "            \n",
    "    def forward(self, x):\n",
    "        x = self.stem(x)\n",
    "\n",
    "        f2 = self.fire2(x)\n",
    "        f3 = self.fire3(f2) + f2\n",
    "        f4 = self.fire4(f3)\n",
    "        f4 = self.maxpool(f4)\n",
    "\n",
    "        f5 = self.fire5(f4) + f4\n",
    "        f6 = self.fire6(f5)\n",
    "        f7 = self.fire7(f6) + f6\n",
    "        f8 = self.fire8(f7)\n",
    "        f8 = self.maxpool(f8)\n",
    "\n",
    "        f9 = self.fire9(f8)\n",
    "        c10 = self.conv10(f9)\n",
    "\n",
    "        x = self.avg(c10)\n",
    "        x = x.view(x.size(0), -1)\n",
    "\n",
    "        return x\n",
    "\n",
    "def squeezenet(class_num=10, activation = 'relu'):\n",
    "    return SqueezeNet(class_num=class_num, activation = activation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to ./cifar-10-python.tar.gz\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "0it [00:00, ?it/s]\u001b[A\n",
      "  0%|          | 0/170498071 [00:00<?, ?it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed download. Trying https -> http instead. Downloading http://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to ./cifar-10-python.tar.gz\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "  0%|          | 49152/170498071 [00:00<10:34, 268663.19it/s]\u001b[A\n",
      "  0%|          | 221184/170498071 [00:00<08:18, 341891.47it/s]\u001b[A\n",
      "  1%|          | 917504/170498071 [00:00<06:00, 470295.30it/s]\u001b[A\n",
      "  2%|▏         | 2809856/170498071 [00:00<04:12, 664769.02it/s]\u001b[A\n",
      "  3%|▎         | 5652480/170498071 [00:00<02:55, 940246.23it/s]\u001b[A\n",
      "  5%|▌         | 9191424/170498071 [00:01<02:01, 1328080.50it/s]\u001b[A\n",
      "  7%|▋         | 12427264/170498071 [00:01<01:25, 1844940.70it/s]\u001b[A\n",
      "  9%|▉         | 16048128/170498071 [00:01<00:59, 2579133.40it/s]\u001b[A\n",
      " 12%|█▏        | 19685376/170498071 [00:01<00:42, 3575732.25it/s]\u001b[A\n",
      " 14%|█▎        | 23404544/170498071 [00:01<00:29, 4905901.33it/s]\u001b[A\n",
      " 16%|█▌        | 27115520/170498071 [00:01<00:21, 6632470.15it/s]\u001b[A\n",
      " 18%|█▊        | 30400512/170498071 [00:01<00:16, 8310936.62it/s]\u001b[A\n",
      " 20%|█▉        | 33939456/170498071 [00:01<00:12, 10786958.76it/s]\u001b[A\n",
      " 22%|██▏       | 37650432/170498071 [00:01<00:09, 13702834.45it/s]\u001b[A\n",
      " 24%|██▍       | 41345024/170498071 [00:02<00:07, 16889176.72it/s]\u001b[A\n",
      " 26%|██▋       | 45015040/170498071 [00:02<00:06, 20149044.10it/s]\u001b[A\n",
      " 28%|██▊       | 48504832/170498071 [00:02<00:05, 20468706.25it/s]\u001b[A\n",
      " 31%|███       | 52183040/170498071 [00:02<00:05, 23603908.19it/s]\u001b[A\n",
      " 33%|███▎      | 55877632/170498071 [00:02<00:04, 26466654.66it/s]\u001b[A\n",
      " 35%|███▍      | 59572224/170498071 [00:02<00:03, 28926111.63it/s]\u001b[A\n",
      " 37%|███▋      | 63053824/170498071 [00:02<00:04, 26092219.15it/s]\u001b[A\n",
      " 39%|███▉      | 66510848/170498071 [00:02<00:03, 28161808.23it/s]\u001b[A\n",
      " 41%|████      | 70197248/170498071 [00:02<00:03, 30303833.68it/s]\u001b[A\n",
      " 43%|████▎     | 73867264/170498071 [00:03<00:03, 31973884.66it/s]\u001b[A\n",
      " 45%|████▌     | 77561856/170498071 [00:03<00:02, 33318003.12it/s]\u001b[A\n",
      " 48%|████▊     | 81076224/170498071 [00:03<00:03, 28531033.67it/s]\u001b[A\n",
      " 50%|████▉     | 84803584/170498071 [00:03<00:02, 30686402.08it/s]\u001b[A\n",
      " 52%|█████▏    | 88178688/170498071 [00:03<00:02, 31540941.64it/s]\u001b[A\n",
      " 54%|█████▍    | 91758592/170498071 [00:03<00:02, 32706352.47it/s]\u001b[A\n",
      " 56%|█████▌    | 95404032/170498071 [00:03<00:02, 33746046.81it/s]\u001b[A\n",
      " 58%|█████▊    | 98877440/170498071 [00:03<00:02, 28928059.77it/s]\u001b[A\n",
      " 60%|██████    | 102580224/170498071 [00:04<00:02, 30957732.72it/s]\u001b[A\n",
      " 62%|██████▏   | 105963520/170498071 [00:04<00:02, 31725552.45it/s]\u001b[A\n",
      " 64%|██████▍   | 109584384/170498071 [00:04<00:01, 32944345.56it/s]\u001b[A\n",
      " 66%|██████▋   | 113197056/170498071 [00:04<00:01, 33833725.67it/s]\u001b[A\n",
      " 68%|██████▊   | 116662272/170498071 [00:04<00:01, 29007593.01it/s]\u001b[A\n",
      " 71%|███████   | 120381440/170498071 [00:04<00:01, 31049463.54it/s]\u001b[A\n",
      " 73%|███████▎  | 123650048/170498071 [00:04<00:01, 30989908.45it/s]\u001b[A\n",
      " 74%|███████▍  | 126943232/170498071 [00:04<00:01, 31486356.13it/s]\u001b[A\n",
      " 76%|███████▋  | 130179072/170498071 [00:04<00:01, 27627337.98it/s]\u001b[A\n",
      " 78%|███████▊  | 133668864/170498071 [00:05<00:01, 29467810.75it/s]\u001b[A\n",
      " 81%|████████  | 137347072/170498071 [00:05<00:01, 31336194.09it/s]\u001b[A\n",
      " 83%|████████▎ | 141025280/170498071 [00:05<00:00, 32791413.88it/s]\u001b[A\n",
      " 85%|████████▍ | 144613376/170498071 [00:05<00:00, 33657484.38it/s]\u001b[A\n",
      " 87%|████████▋ | 148250624/170498071 [00:05<00:00, 34426781.04it/s]\u001b[A\n",
      " 89%|████████▉ | 151756800/170498071 [00:05<00:00, 29160406.58it/s]\u001b[A\n",
      " 91%|█████████ | 155123712/170498071 [00:05<00:00, 30367601.72it/s]\u001b[A\n",
      " 93%|█████████▎| 158580736/170498071 [00:05<00:00, 31511358.15it/s]\u001b[A\n",
      " 95%|█████████▌| 162275328/170498071 [00:05<00:00, 32965228.10it/s]\u001b[A\n",
      " 97%|█████████▋| 165978112/170498071 [00:05<00:00, 34080429.75it/s]\u001b[A\n",
      " 99%|█████████▉| 169607168/170498071 [00:06<00:00, 34707396.25it/s]\u001b[A\n",
      "170500096it [00:06, 27636343.11it/s]                               \u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "trainloader = get_training_dataloader(train_transform)\n",
    "testloader = get_testing_dataloader(test_transform)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda', index=0)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "epochs = 100\n",
    "batch_size = 128\n",
    "learning_rate = 0.001\n",
    "device = torch.device('cuda:0' if torch.cuda.is_available() else \"cpu\")\n",
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = squeezenet(activation = 'mish')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set loss function\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "# set optimizer, only train the classifier parameters, feature parameters are frozen\n",
    "optimizer = Adam(model.parameters(), lr=learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_stats = pd.DataFrame(columns = ['Epoch', 'Time per epoch', 'Avg time per step', 'Train loss', 'Train accuracy', 'Train top-3 accuracy','Test loss', 'Test accuracy', 'Test top-3 accuracy']) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100.. Time per epoch: 25.9280.. Average time per step: 0.0663.. Train loss: 1.4351.. Train accuracy: 0.4721.. Top-3 train accuracy: 0.7985.. Test loss: 1.2524.. Test accuracy: 0.5464.. Top-3 test accuracy: 0.8628\n",
      "Epoch 2/100.. Time per epoch: 25.7514.. Average time per step: 0.0659.. Train loss: 1.1094.. Train accuracy: 0.6017.. Top-3 train accuracy: 0.8797.. Test loss: 0.9673.. Test accuracy: 0.6541.. Top-3 test accuracy: 0.9066\n",
      "Epoch 3/100.. Time per epoch: 25.7034.. Average time per step: 0.0657.. Train loss: 0.9516.. Train accuracy: 0.6612.. Top-3 train accuracy: 0.9084.. Test loss: 0.8619.. Test accuracy: 0.7007.. Top-3 test accuracy: 0.9249\n",
      "Epoch 4/100.. Time per epoch: 25.5914.. Average time per step: 0.0655.. Train loss: 0.8612.. Train accuracy: 0.6946.. Top-3 train accuracy: 0.9219.. Test loss: 0.7503.. Test accuracy: 0.7359.. Top-3 test accuracy: 0.9405\n",
      "Epoch 5/100.. Time per epoch: 25.7364.. Average time per step: 0.0658.. Train loss: 0.7842.. Train accuracy: 0.7240.. Top-3 train accuracy: 0.9325.. Test loss: 0.7054.. Test accuracy: 0.7573.. Top-3 test accuracy: 0.9471\n",
      "Epoch 6/100.. Time per epoch: 25.8059.. Average time per step: 0.0660.. Train loss: 0.7328.. Train accuracy: 0.7425.. Top-3 train accuracy: 0.9398.. Test loss: 0.6974.. Test accuracy: 0.7536.. Top-3 test accuracy: 0.9491\n",
      "Epoch 7/100.. Time per epoch: 25.7488.. Average time per step: 0.0659.. Train loss: 0.6879.. Train accuracy: 0.7606.. Top-3 train accuracy: 0.9449.. Test loss: 0.6006.. Test accuracy: 0.7933.. Top-3 test accuracy: 0.9600\n",
      "Epoch 8/100.. Time per epoch: 25.7246.. Average time per step: 0.0658.. Train loss: 0.6522.. Train accuracy: 0.7733.. Top-3 train accuracy: 0.9508.. Test loss: 0.5901.. Test accuracy: 0.8004.. Top-3 test accuracy: 0.9594\n",
      "Epoch 9/100.. Time per epoch: 26.4283.. Average time per step: 0.0676.. Train loss: 0.6163.. Train accuracy: 0.7849.. Top-3 train accuracy: 0.9538.. Test loss: 0.5602.. Test accuracy: 0.8076.. Top-3 test accuracy: 0.9663\n",
      "Epoch 10/100.. Time per epoch: 25.8959.. Average time per step: 0.0662.. Train loss: 0.5943.. Train accuracy: 0.7903.. Top-3 train accuracy: 0.9573.. Test loss: 0.5923.. Test accuracy: 0.8013.. Top-3 test accuracy: 0.9565\n",
      "Epoch 11/100.. Time per epoch: 25.8385.. Average time per step: 0.0661.. Train loss: 0.5678.. Train accuracy: 0.8006.. Top-3 train accuracy: 0.9605.. Test loss: 0.5275.. Test accuracy: 0.8190.. Top-3 test accuracy: 0.9679\n",
      "Epoch 12/100.. Time per epoch: 25.9165.. Average time per step: 0.0663.. Train loss: 0.5486.. Train accuracy: 0.8103.. Top-3 train accuracy: 0.9624.. Test loss: 0.5327.. Test accuracy: 0.8173.. Top-3 test accuracy: 0.9677\n",
      "Epoch 13/100.. Time per epoch: 25.9257.. Average time per step: 0.0663.. Train loss: 0.5228.. Train accuracy: 0.8177.. Top-3 train accuracy: 0.9658.. Test loss: 0.5210.. Test accuracy: 0.8233.. Top-3 test accuracy: 0.9623\n",
      "Epoch 14/100.. Time per epoch: 25.8143.. Average time per step: 0.0660.. Train loss: 0.5091.. Train accuracy: 0.8230.. Top-3 train accuracy: 0.9670.. Test loss: 0.4857.. Test accuracy: 0.8348.. Top-3 test accuracy: 0.9706\n",
      "Epoch 15/100.. Time per epoch: 25.8614.. Average time per step: 0.0661.. Train loss: 0.4905.. Train accuracy: 0.8266.. Top-3 train accuracy: 0.9686.. Test loss: 0.4779.. Test accuracy: 0.8343.. Top-3 test accuracy: 0.9717\n",
      "Epoch 16/100.. Time per epoch: 25.6232.. Average time per step: 0.0655.. Train loss: 0.4746.. Train accuracy: 0.8327.. Top-3 train accuracy: 0.9706.. Test loss: 0.4796.. Test accuracy: 0.8331.. Top-3 test accuracy: 0.9711\n",
      "Epoch 17/100.. Time per epoch: 25.6938.. Average time per step: 0.0657.. Train loss: 0.4592.. Train accuracy: 0.8389.. Top-3 train accuracy: 0.9725.. Test loss: 0.4614.. Test accuracy: 0.8428.. Top-3 test accuracy: 0.9733\n",
      "Epoch 18/100.. Time per epoch: 25.6333.. Average time per step: 0.0656.. Train loss: 0.4465.. Train accuracy: 0.8429.. Top-3 train accuracy: 0.9731.. Test loss: 0.4484.. Test accuracy: 0.8487.. Top-3 test accuracy: 0.9728\n",
      "Epoch 19/100.. Time per epoch: 25.5068.. Average time per step: 0.0652.. Train loss: 0.4301.. Train accuracy: 0.8499.. Top-3 train accuracy: 0.9749.. Test loss: 0.4656.. Test accuracy: 0.8403.. Top-3 test accuracy: 0.9726\n",
      "Epoch 20/100.. Time per epoch: 25.7227.. Average time per step: 0.0658.. Train loss: 0.4223.. Train accuracy: 0.8510.. Top-3 train accuracy: 0.9761.. Test loss: 0.4550.. Test accuracy: 0.8430.. Top-3 test accuracy: 0.9736\n",
      "Epoch 21/100.. Time per epoch: 25.7428.. Average time per step: 0.0658.. Train loss: 0.4074.. Train accuracy: 0.8582.. Top-3 train accuracy: 0.9767.. Test loss: 0.4265.. Test accuracy: 0.8534.. Top-3 test accuracy: 0.9752\n",
      "Epoch 22/100.. Time per epoch: 25.6477.. Average time per step: 0.0656.. Train loss: 0.3926.. Train accuracy: 0.8639.. Top-3 train accuracy: 0.9785.. Test loss: 0.4291.. Test accuracy: 0.8563.. Top-3 test accuracy: 0.9758\n",
      "Epoch 23/100.. Time per epoch: 25.8011.. Average time per step: 0.0660.. Train loss: 0.3851.. Train accuracy: 0.8650.. Top-3 train accuracy: 0.9794.. Test loss: 0.4177.. Test accuracy: 0.8576.. Top-3 test accuracy: 0.9775\n",
      "Epoch 24/100.. Time per epoch: 25.8386.. Average time per step: 0.0661.. Train loss: 0.3748.. Train accuracy: 0.8684.. Top-3 train accuracy: 0.9800.. Test loss: 0.4308.. Test accuracy: 0.8551.. Top-3 test accuracy: 0.9766\n",
      "Epoch 25/100.. Time per epoch: 25.7385.. Average time per step: 0.0658.. Train loss: 0.3660.. Train accuracy: 0.8713.. Top-3 train accuracy: 0.9808.. Test loss: 0.4362.. Test accuracy: 0.8552.. Top-3 test accuracy: 0.9763\n",
      "Epoch 26/100.. Time per epoch: 25.8730.. Average time per step: 0.0662.. Train loss: 0.3564.. Train accuracy: 0.8756.. Top-3 train accuracy: 0.9814.. Test loss: 0.4285.. Test accuracy: 0.8597.. Top-3 test accuracy: 0.9763\n",
      "Epoch 27/100.. Time per epoch: 25.6726.. Average time per step: 0.0657.. Train loss: 0.3436.. Train accuracy: 0.8784.. Top-3 train accuracy: 0.9830.. Test loss: 0.4348.. Test accuracy: 0.8540.. Top-3 test accuracy: 0.9760\n",
      "Epoch 28/100.. Time per epoch: 25.7702.. Average time per step: 0.0659.. Train loss: 0.3320.. Train accuracy: 0.8840.. Top-3 train accuracy: 0.9843.. Test loss: 0.4135.. Test accuracy: 0.8660.. Top-3 test accuracy: 0.9780\n",
      "Epoch 29/100.. Time per epoch: 25.7841.. Average time per step: 0.0659.. Train loss: 0.3327.. Train accuracy: 0.8827.. Top-3 train accuracy: 0.9830.. Test loss: 0.4065.. Test accuracy: 0.8696.. Top-3 test accuracy: 0.9781\n",
      "Epoch 30/100.. Time per epoch: 25.7832.. Average time per step: 0.0659.. Train loss: 0.3193.. Train accuracy: 0.8889.. Top-3 train accuracy: 0.9851.. Test loss: 0.4125.. Test accuracy: 0.8625.. Top-3 test accuracy: 0.9814\n",
      "Epoch 31/100.. Time per epoch: 25.8060.. Average time per step: 0.0660.. Train loss: 0.3115.. Train accuracy: 0.8896.. Top-3 train accuracy: 0.9861.. Test loss: 0.4154.. Test accuracy: 0.8612.. Top-3 test accuracy: 0.9789\n",
      "Epoch 32/100.. Time per epoch: 26.0847.. Average time per step: 0.0667.. Train loss: 0.3092.. Train accuracy: 0.8905.. Top-3 train accuracy: 0.9857.. Test loss: 0.4167.. Test accuracy: 0.8630.. Top-3 test accuracy: 0.9789\n",
      "Epoch 33/100.. Time per epoch: 25.9233.. Average time per step: 0.0663.. Train loss: 0.2968.. Train accuracy: 0.8942.. Top-3 train accuracy: 0.9874.. Test loss: 0.4102.. Test accuracy: 0.8658.. Top-3 test accuracy: 0.9790\n",
      "Epoch 34/100.. Time per epoch: 25.9390.. Average time per step: 0.0663.. Train loss: 0.2850.. Train accuracy: 0.8995.. Top-3 train accuracy: 0.9879.. Test loss: 0.4054.. Test accuracy: 0.8694.. Top-3 test accuracy: 0.9790\n",
      "Epoch 35/100.. Time per epoch: 26.0816.. Average time per step: 0.0667.. Train loss: 0.2815.. Train accuracy: 0.9002.. Top-3 train accuracy: 0.9882.. Test loss: 0.4015.. Test accuracy: 0.8709.. Top-3 test accuracy: 0.9809\n",
      "Epoch 36/100.. Time per epoch: 25.9276.. Average time per step: 0.0663.. Train loss: 0.2753.. Train accuracy: 0.9026.. Top-3 train accuracy: 0.9887.. Test loss: 0.4390.. Test accuracy: 0.8621.. Top-3 test accuracy: 0.9770\n",
      "Epoch 37/100.. Time per epoch: 25.8202.. Average time per step: 0.0660.. Train loss: 0.2710.. Train accuracy: 0.9040.. Top-3 train accuracy: 0.9884.. Test loss: 0.4048.. Test accuracy: 0.8724.. Top-3 test accuracy: 0.9796\n",
      "Epoch 38/100.. Time per epoch: 25.5849.. Average time per step: 0.0654.. Train loss: 0.2645.. Train accuracy: 0.9073.. Top-3 train accuracy: 0.9898.. Test loss: 0.4061.. Test accuracy: 0.8738.. Top-3 test accuracy: 0.9799\n",
      "Epoch 39/100.. Time per epoch: 25.5480.. Average time per step: 0.0653.. Train loss: 0.2588.. Train accuracy: 0.9073.. Top-3 train accuracy: 0.9904.. Test loss: 0.4290.. Test accuracy: 0.8647.. Top-3 test accuracy: 0.9799\n",
      "Epoch 40/100.. Time per epoch: 25.6777.. Average time per step: 0.0657.. Train loss: 0.2533.. Train accuracy: 0.9103.. Top-3 train accuracy: 0.9902.. Test loss: 0.4201.. Test accuracy: 0.8659.. Top-3 test accuracy: 0.9791\n",
      "Epoch 41/100.. Time per epoch: 25.5869.. Average time per step: 0.0654.. Train loss: 0.2461.. Train accuracy: 0.9133.. Top-3 train accuracy: 0.9908.. Test loss: 0.4088.. Test accuracy: 0.8712.. Top-3 test accuracy: 0.9800\n",
      "Epoch 42/100.. Time per epoch: 25.5274.. Average time per step: 0.0653.. Train loss: 0.2443.. Train accuracy: 0.9132.. Top-3 train accuracy: 0.9905.. Test loss: 0.4177.. Test accuracy: 0.8691.. Top-3 test accuracy: 0.9817\n",
      "Epoch 43/100.. Time per epoch: 25.7983.. Average time per step: 0.0660.. Train loss: 0.2377.. Train accuracy: 0.9152.. Top-3 train accuracy: 0.9909.. Test loss: 0.4353.. Test accuracy: 0.8664.. Top-3 test accuracy: 0.9795\n",
      "Epoch 44/100.. Time per epoch: 25.6421.. Average time per step: 0.0656.. Train loss: 0.2341.. Train accuracy: 0.9163.. Top-3 train accuracy: 0.9919.. Test loss: 0.3964.. Test accuracy: 0.8731.. Top-3 test accuracy: 0.9824\n",
      "Epoch 45/100.. Time per epoch: 25.7545.. Average time per step: 0.0659.. Train loss: 0.2259.. Train accuracy: 0.9207.. Top-3 train accuracy: 0.9918.. Test loss: 0.4221.. Test accuracy: 0.8663.. Top-3 test accuracy: 0.9784\n",
      "Epoch 46/100.. Time per epoch: 25.9669.. Average time per step: 0.0664.. Train loss: 0.2204.. Train accuracy: 0.9221.. Top-3 train accuracy: 0.9923.. Test loss: 0.4380.. Test accuracy: 0.8691.. Top-3 test accuracy: 0.9779\n",
      "Epoch 47/100.. Time per epoch: 25.6555.. Average time per step: 0.0656.. Train loss: 0.2187.. Train accuracy: 0.9231.. Top-3 train accuracy: 0.9925.. Test loss: 0.4158.. Test accuracy: 0.8712.. Top-3 test accuracy: 0.9815\n",
      "Epoch 48/100.. Time per epoch: 25.6120.. Average time per step: 0.0655.. Train loss: 0.2144.. Train accuracy: 0.9233.. Top-3 train accuracy: 0.9929.. Test loss: 0.4055.. Test accuracy: 0.8732.. Top-3 test accuracy: 0.9808\n",
      "Epoch 49/100.. Time per epoch: 25.7407.. Average time per step: 0.0658.. Train loss: 0.2099.. Train accuracy: 0.9259.. Top-3 train accuracy: 0.9935.. Test loss: 0.4316.. Test accuracy: 0.8695.. Top-3 test accuracy: 0.9804\n",
      "Epoch 50/100.. Time per epoch: 25.5810.. Average time per step: 0.0654.. Train loss: 0.2063.. Train accuracy: 0.9255.. Top-3 train accuracy: 0.9931.. Test loss: 0.4325.. Test accuracy: 0.8712.. Top-3 test accuracy: 0.9788\n",
      "Epoch 51/100.. Time per epoch: 25.6392.. Average time per step: 0.0656.. Train loss: 0.2006.. Train accuracy: 0.9285.. Top-3 train accuracy: 0.9937.. Test loss: 0.4060.. Test accuracy: 0.8777.. Top-3 test accuracy: 0.9820\n",
      "Epoch 52/100.. Time per epoch: 25.6804.. Average time per step: 0.0657.. Train loss: 0.1958.. Train accuracy: 0.9298.. Top-3 train accuracy: 0.9943.. Test loss: 0.4081.. Test accuracy: 0.8789.. Top-3 test accuracy: 0.9828\n",
      "Epoch 53/100.. Time per epoch: 25.6951.. Average time per step: 0.0657.. Train loss: 0.1915.. Train accuracy: 0.9315.. Top-3 train accuracy: 0.9939.. Test loss: 0.4233.. Test accuracy: 0.8735.. Top-3 test accuracy: 0.9800\n",
      "Epoch 54/100.. Time per epoch: 25.7656.. Average time per step: 0.0659.. Train loss: 0.1884.. Train accuracy: 0.9337.. Top-3 train accuracy: 0.9945.. Test loss: 0.4186.. Test accuracy: 0.8753.. Top-3 test accuracy: 0.9822\n",
      "Epoch 55/100.. Time per epoch: 25.5518.. Average time per step: 0.0653.. Train loss: 0.1859.. Train accuracy: 0.9330.. Top-3 train accuracy: 0.9947.. Test loss: 0.4225.. Test accuracy: 0.8755.. Top-3 test accuracy: 0.9792\n",
      "Epoch 56/100.. Time per epoch: 25.7438.. Average time per step: 0.0658.. Train loss: 0.1809.. Train accuracy: 0.9361.. Top-3 train accuracy: 0.9947.. Test loss: 0.4316.. Test accuracy: 0.8730.. Top-3 test accuracy: 0.9804\n",
      "Epoch 57/100.. Time per epoch: 25.7704.. Average time per step: 0.0659.. Train loss: 0.1752.. Train accuracy: 0.9379.. Top-3 train accuracy: 0.9952.. Test loss: 0.4235.. Test accuracy: 0.8795.. Top-3 test accuracy: 0.9794\n",
      "Epoch 58/100.. Time per epoch: 25.6818.. Average time per step: 0.0657.. Train loss: 0.1719.. Train accuracy: 0.9392.. Top-3 train accuracy: 0.9954.. Test loss: 0.4322.. Test accuracy: 0.8773.. Top-3 test accuracy: 0.9796\n",
      "Epoch 59/100.. Time per epoch: 25.5751.. Average time per step: 0.0654.. Train loss: 0.1760.. Train accuracy: 0.9371.. Top-3 train accuracy: 0.9951.. Test loss: 0.4175.. Test accuracy: 0.8774.. Top-3 test accuracy: 0.9804\n",
      "Epoch 60/100.. Time per epoch: 25.6764.. Average time per step: 0.0657.. Train loss: 0.1710.. Train accuracy: 0.9400.. Top-3 train accuracy: 0.9953.. Test loss: 0.4436.. Test accuracy: 0.8728.. Top-3 test accuracy: 0.9799\n",
      "Epoch 61/100.. Time per epoch: 25.5278.. Average time per step: 0.0653.. Train loss: 0.1660.. Train accuracy: 0.9408.. Top-3 train accuracy: 0.9956.. Test loss: 0.4394.. Test accuracy: 0.8772.. Top-3 test accuracy: 0.9805\n",
      "Epoch 62/100.. Time per epoch: 25.5495.. Average time per step: 0.0653.. Train loss: 0.1609.. Train accuracy: 0.9427.. Top-3 train accuracy: 0.9959.. Test loss: 0.4219.. Test accuracy: 0.8809.. Top-3 test accuracy: 0.9805\n",
      "Epoch 63/100.. Time per epoch: 25.6605.. Average time per step: 0.0656.. Train loss: 0.1606.. Train accuracy: 0.9415.. Top-3 train accuracy: 0.9961.. Test loss: 0.4518.. Test accuracy: 0.8704.. Top-3 test accuracy: 0.9802\n",
      "Epoch 64/100.. Time per epoch: 25.6931.. Average time per step: 0.0657.. Train loss: 0.1573.. Train accuracy: 0.9438.. Top-3 train accuracy: 0.9960.. Test loss: 0.4438.. Test accuracy: 0.8736.. Top-3 test accuracy: 0.9791\n",
      "Epoch 65/100.. Time per epoch: 25.6811.. Average time per step: 0.0657.. Train loss: 0.1533.. Train accuracy: 0.9450.. Top-3 train accuracy: 0.9961.. Test loss: 0.4277.. Test accuracy: 0.8802.. Top-3 test accuracy: 0.9829\n",
      "Epoch 66/100.. Time per epoch: 25.7094.. Average time per step: 0.0658.. Train loss: 0.1525.. Train accuracy: 0.9456.. Top-3 train accuracy: 0.9966.. Test loss: 0.4465.. Test accuracy: 0.8744.. Top-3 test accuracy: 0.9785\n",
      "Epoch 67/100.. Time per epoch: 25.7180.. Average time per step: 0.0658.. Train loss: 0.1454.. Train accuracy: 0.9481.. Top-3 train accuracy: 0.9965.. Test loss: 0.4805.. Test accuracy: 0.8709.. Top-3 test accuracy: 0.9777\n",
      "Epoch 68/100.. Time per epoch: 25.7761.. Average time per step: 0.0659.. Train loss: 0.1430.. Train accuracy: 0.9495.. Top-3 train accuracy: 0.9968.. Test loss: 0.4531.. Test accuracy: 0.8793.. Top-3 test accuracy: 0.9812\n",
      "Epoch 69/100.. Time per epoch: 25.6429.. Average time per step: 0.0656.. Train loss: 0.1475.. Train accuracy: 0.9480.. Top-3 train accuracy: 0.9967.. Test loss: 0.4479.. Test accuracy: 0.8741.. Top-3 test accuracy: 0.9809\n",
      "Epoch 70/100.. Time per epoch: 25.4831.. Average time per step: 0.0652.. Train loss: 0.1468.. Train accuracy: 0.9487.. Top-3 train accuracy: 0.9966.. Test loss: 0.4308.. Test accuracy: 0.8802.. Top-3 test accuracy: 0.9810\n",
      "Epoch 71/100.. Time per epoch: 25.6725.. Average time per step: 0.0657.. Train loss: 0.1404.. Train accuracy: 0.9493.. Top-3 train accuracy: 0.9970.. Test loss: 0.4544.. Test accuracy: 0.8733.. Top-3 test accuracy: 0.9816\n",
      "Epoch 72/100.. Time per epoch: 25.6292.. Average time per step: 0.0655.. Train loss: 0.1336.. Train accuracy: 0.9523.. Top-3 train accuracy: 0.9971.. Test loss: 0.4548.. Test accuracy: 0.8796.. Top-3 test accuracy: 0.9805\n",
      "Epoch 73/100.. Time per epoch: 25.7004.. Average time per step: 0.0657.. Train loss: 0.1384.. Train accuracy: 0.9503.. Top-3 train accuracy: 0.9968.. Test loss: 0.4540.. Test accuracy: 0.8786.. Top-3 test accuracy: 0.9802\n",
      "Epoch 74/100.. Time per epoch: 25.7879.. Average time per step: 0.0660.. Train loss: 0.1315.. Train accuracy: 0.9533.. Top-3 train accuracy: 0.9975.. Test loss: 0.4541.. Test accuracy: 0.8806.. Top-3 test accuracy: 0.9804\n",
      "Epoch 75/100.. Time per epoch: 25.8224.. Average time per step: 0.0660.. Train loss: 0.1366.. Train accuracy: 0.9518.. Top-3 train accuracy: 0.9971.. Test loss: 0.4500.. Test accuracy: 0.8773.. Top-3 test accuracy: 0.9813\n",
      "Epoch 76/100.. Time per epoch: 25.6701.. Average time per step: 0.0657.. Train loss: 0.1261.. Train accuracy: 0.9551.. Top-3 train accuracy: 0.9974.. Test loss: 0.4694.. Test accuracy: 0.8792.. Top-3 test accuracy: 0.9820\n",
      "Epoch 77/100.. Time per epoch: 25.7373.. Average time per step: 0.0658.. Train loss: 0.1291.. Train accuracy: 0.9543.. Top-3 train accuracy: 0.9975.. Test loss: 0.4558.. Test accuracy: 0.8798.. Top-3 test accuracy: 0.9808\n",
      "Epoch 78/100.. Time per epoch: 25.7404.. Average time per step: 0.0658.. Train loss: 0.1248.. Train accuracy: 0.9554.. Top-3 train accuracy: 0.9976.. Test loss: 0.4806.. Test accuracy: 0.8721.. Top-3 test accuracy: 0.9822\n",
      "Epoch 79/100.. Time per epoch: 25.9021.. Average time per step: 0.0662.. Train loss: 0.1245.. Train accuracy: 0.9557.. Top-3 train accuracy: 0.9973.. Test loss: 0.4893.. Test accuracy: 0.8743.. Top-3 test accuracy: 0.9811\n",
      "Epoch 80/100.. Time per epoch: 25.7668.. Average time per step: 0.0659.. Train loss: 0.1240.. Train accuracy: 0.9568.. Top-3 train accuracy: 0.9975.. Test loss: 0.4377.. Test accuracy: 0.8823.. Top-3 test accuracy: 0.9820\n",
      "Epoch 81/100.. Time per epoch: 25.6729.. Average time per step: 0.0657.. Train loss: 0.1185.. Train accuracy: 0.9572.. Top-3 train accuracy: 0.9976.. Test loss: 0.4610.. Test accuracy: 0.8754.. Top-3 test accuracy: 0.9816\n",
      "Epoch 82/100.. Time per epoch: 25.7638.. Average time per step: 0.0659.. Train loss: 0.1228.. Train accuracy: 0.9569.. Top-3 train accuracy: 0.9972.. Test loss: 0.4497.. Test accuracy: 0.8803.. Top-3 test accuracy: 0.9822\n",
      "Epoch 83/100.. Time per epoch: 25.8720.. Average time per step: 0.0662.. Train loss: 0.1147.. Train accuracy: 0.9594.. Top-3 train accuracy: 0.9978.. Test loss: 0.4507.. Test accuracy: 0.8823.. Top-3 test accuracy: 0.9808\n",
      "Epoch 84/100.. Time per epoch: 25.7542.. Average time per step: 0.0659.. Train loss: 0.1152.. Train accuracy: 0.9584.. Top-3 train accuracy: 0.9979.. Test loss: 0.4503.. Test accuracy: 0.8824.. Top-3 test accuracy: 0.9826\n",
      "Epoch 85/100.. Time per epoch: 25.7215.. Average time per step: 0.0658.. Train loss: 0.1106.. Train accuracy: 0.9607.. Top-3 train accuracy: 0.9981.. Test loss: 0.4708.. Test accuracy: 0.8822.. Top-3 test accuracy: 0.9792\n",
      "Epoch 86/100.. Time per epoch: 25.6460.. Average time per step: 0.0656.. Train loss: 0.1173.. Train accuracy: 0.9588.. Top-3 train accuracy: 0.9979.. Test loss: 0.4608.. Test accuracy: 0.8837.. Top-3 test accuracy: 0.9836\n",
      "Epoch 87/100.. Time per epoch: 25.5864.. Average time per step: 0.0654.. Train loss: 0.1091.. Train accuracy: 0.9614.. Top-3 train accuracy: 0.9983.. Test loss: 0.4766.. Test accuracy: 0.8783.. Top-3 test accuracy: 0.9809\n",
      "Epoch 88/100.. Time per epoch: 25.6139.. Average time per step: 0.0655.. Train loss: 0.1070.. Train accuracy: 0.9624.. Top-3 train accuracy: 0.9982.. Test loss: 0.4806.. Test accuracy: 0.8785.. Top-3 test accuracy: 0.9797\n",
      "Epoch 89/100.. Time per epoch: 25.6921.. Average time per step: 0.0657.. Train loss: 0.1055.. Train accuracy: 0.9621.. Top-3 train accuracy: 0.9983.. Test loss: 0.4740.. Test accuracy: 0.8809.. Top-3 test accuracy: 0.9789\n",
      "Epoch 90/100.. Time per epoch: 25.7787.. Average time per step: 0.0659.. Train loss: 0.1066.. Train accuracy: 0.9620.. Top-3 train accuracy: 0.9981.. Test loss: 0.4564.. Test accuracy: 0.8834.. Top-3 test accuracy: 0.9813\n",
      "Epoch 91/100.. Time per epoch: 25.5721.. Average time per step: 0.0654.. Train loss: 0.1114.. Train accuracy: 0.9609.. Top-3 train accuracy: 0.9977.. Test loss: 0.4660.. Test accuracy: 0.8825.. Top-3 test accuracy: 0.9823\n",
      "Epoch 92/100.. Time per epoch: 25.6893.. Average time per step: 0.0657.. Train loss: 0.1036.. Train accuracy: 0.9624.. Top-3 train accuracy: 0.9981.. Test loss: 0.4876.. Test accuracy: 0.8778.. Top-3 test accuracy: 0.9829\n",
      "Epoch 93/100.. Time per epoch: 25.5311.. Average time per step: 0.0653.. Train loss: 0.1004.. Train accuracy: 0.9648.. Top-3 train accuracy: 0.9984.. Test loss: 0.4901.. Test accuracy: 0.8772.. Top-3 test accuracy: 0.9821\n",
      "Epoch 94/100.. Time per epoch: 25.5897.. Average time per step: 0.0654.. Train loss: 0.0974.. Train accuracy: 0.9651.. Top-3 train accuracy: 0.9987.. Test loss: 0.4728.. Test accuracy: 0.8838.. Top-3 test accuracy: 0.9815\n",
      "Epoch 95/100.. Time per epoch: 25.5807.. Average time per step: 0.0654.. Train loss: 0.1021.. Train accuracy: 0.9638.. Top-3 train accuracy: 0.9978.. Test loss: 0.4850.. Test accuracy: 0.8815.. Top-3 test accuracy: 0.9804\n",
      "Epoch 96/100.. Time per epoch: 25.6582.. Average time per step: 0.0656.. Train loss: 0.0977.. Train accuracy: 0.9653.. Top-3 train accuracy: 0.9987.. Test loss: 0.5002.. Test accuracy: 0.8791.. Top-3 test accuracy: 0.9795\n",
      "Epoch 97/100.. Time per epoch: 25.6807.. Average time per step: 0.0657.. Train loss: 0.0965.. Train accuracy: 0.9655.. Top-3 train accuracy: 0.9984.. Test loss: 0.4808.. Test accuracy: 0.8847.. Top-3 test accuracy: 0.9815\n",
      "Epoch 98/100.. Time per epoch: 25.5548.. Average time per step: 0.0654.. Train loss: 0.0964.. Train accuracy: 0.9658.. Top-3 train accuracy: 0.9984.. Test loss: 0.4952.. Test accuracy: 0.8812.. Top-3 test accuracy: 0.9810\n",
      "Epoch 99/100.. Time per epoch: 25.7580.. Average time per step: 0.0659.. Train loss: 0.0952.. Train accuracy: 0.9669.. Top-3 train accuracy: 0.9986.. Test loss: 0.4820.. Test accuracy: 0.8835.. Top-3 test accuracy: 0.9801\n",
      "Epoch 100/100.. Time per epoch: 25.8388.. Average time per step: 0.0661.. Train loss: 0.0908.. Train accuracy: 0.9682.. Top-3 train accuracy: 0.9989.. Test loss: 0.5152.. Test accuracy: 0.8777.. Top-3 test accuracy: 0.9795\n"
     ]
    }
   ],
   "source": [
    "#train the model\n",
    "model.to(device)\n",
    "\n",
    "steps = 0\n",
    "running_loss = 0\n",
    "for epoch in range(epochs):\n",
    "    \n",
    "    since = time.time()\n",
    "    \n",
    "    train_accuracy = 0\n",
    "    top3_train_accuracy = 0 \n",
    "    for inputs, labels in trainloader:\n",
    "        steps += 1\n",
    "        # Move input and label tensors to the default device\n",
    "        inputs, labels = inputs.to(device), labels.to(device)\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        logps = model.forward(inputs)\n",
    "        loss = criterion(logps, labels)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        running_loss += loss.item()\n",
    "        \n",
    "        # calculate train top-1 accuracy\n",
    "        ps = torch.exp(logps)\n",
    "        top_p, top_class = ps.topk(1, dim=1)\n",
    "        equals = top_class == labels.view(*top_class.shape)\n",
    "        train_accuracy += torch.mean(equals.type(torch.FloatTensor)).item()\n",
    "        \n",
    "        # Calculate train top-3 accuracy\n",
    "        np_top3_class = ps.topk(3, dim=1)[1].cpu().numpy()\n",
    "        target_numpy = labels.cpu().numpy()\n",
    "        top3_train_accuracy += np.mean([1 if target_numpy[i] in np_top3_class[i] else 0 for i in range(0, len(target_numpy))])\n",
    "        \n",
    "    time_elapsed = time.time() - since\n",
    "    \n",
    "    test_loss = 0\n",
    "    test_accuracy = 0\n",
    "    top3_test_accuracy = 0\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        for inputs, labels in testloader:\n",
    "            inputs, labels = inputs.to(device), labels.to(device)\n",
    "            logps = model.forward(inputs)\n",
    "            batch_loss = criterion(logps, labels)\n",
    "\n",
    "            test_loss += batch_loss.item()\n",
    "\n",
    "            # Calculate test top-1 accuracy\n",
    "            ps = torch.exp(logps)\n",
    "            top_p, top_class = ps.topk(1, dim=1)\n",
    "            equals = top_class == labels.view(*top_class.shape)\n",
    "            test_accuracy += torch.mean(equals.type(torch.FloatTensor)).item()\n",
    "            \n",
    "            # Calculate test top-3 accuracy\n",
    "            np_top3_class = ps.topk(3, dim=1)[1].cpu().numpy()\n",
    "            target_numpy = labels.cpu().numpy()\n",
    "            top3_test_accuracy += np.mean([1 if target_numpy[i] in np_top3_class[i] else 0 for i in range(0, len(target_numpy))])\n",
    "\n",
    "    print(f\"Epoch {epoch+1}/{epochs}.. \"\n",
    "          f\"Time per epoch: {time_elapsed:.4f}.. \"\n",
    "          f\"Average time per step: {time_elapsed/len(trainloader):.4f}.. \"\n",
    "          f\"Train loss: {running_loss/len(trainloader):.4f}.. \"\n",
    "          f\"Train accuracy: {train_accuracy/len(trainloader):.4f}.. \"\n",
    "          f\"Top-3 train accuracy: {top3_train_accuracy/len(trainloader):.4f}.. \"\n",
    "          f\"Test loss: {test_loss/len(testloader):.4f}.. \"\n",
    "          f\"Test accuracy: {test_accuracy/len(testloader):.4f}.. \"\n",
    "          f\"Top-3 test accuracy: {top3_test_accuracy/len(testloader):.4f}\")\n",
    "\n",
    "    train_stats = train_stats.append({'Epoch': epoch, 'Time per epoch':time_elapsed, 'Avg time per step': time_elapsed/len(trainloader), 'Train loss' : running_loss/len(trainloader), 'Train accuracy': train_accuracy/len(trainloader), 'Train top-3 accuracy':top3_train_accuracy/len(trainloader),'Test loss' : test_loss/len(testloader), 'Test accuracy': test_accuracy/len(testloader), 'Test top-3 accuracy':top3_test_accuracy/len(testloader)}, ignore_index=True)\n",
    "\n",
    "    running_loss = 0\n",
    "    model.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_stats.to_csv('train_log_SqueezeNet_Mish.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
